from jili.core.printlog import print
from pymongo import MongoClient as mc
from jili.data.db import getdb_client,insert_one
import datetime
def gen_key(i,keys):
    key = "k"
    for k in keys:
        tmp = str(i[k])
        tmp = tmp.replace(" ", "")
        key = key + "-" + tmp
    return key
def copydb(sorureurl,tourl,sourcedb,todb,onlydealcus=None,nodealcus=[],isfresh=False,refresh_cu=[],refresh_find={},ischeck=False,isinsert_one=False,renames={},calldeal=None):
    print("start")
    clientk = mc(sorureurl)
    dbk = clientk[sourcedb]
    client = mc(tourl)
    db = client[todb]
    cus=dbk.list_collection_names()
    print("from ",cus)
    tocus=db.list_collection_names()
    print("to ", tocus)
    dealcus={}
    uptime=datetime.datetime.now()
    for i in cus:
        if onlydealcus:
            if i not in onlydealcus:
                continue
        if i in nodealcus:
            continue
        if i in renames.keys():
            ii=renames[i]
        else:
            ii=i
        if i in tocus:
            t = {}
            n0 = db[ii].count_documents(refresh_find)
            n1 = dbk[i].count_documents(refresh_find)
            if n1!=0:
                if refresh_find:
                    isfresh0 = True
                elif i in refresh_cu:
                    isfresh0 = True
                else:
                    isfresh0 = isfresh
                if isfresh0:
                    # t = {}
                    t["需处理"] = n1
                    t["数据源总计"] = n1
                    t["现有"] = n0
                    t["maxid"] = None
                    if refresh_find:
                        for ii in db[ii].find({},{"_id":1}).sort([("_id",-1)]).limit(1):
                            t["maxid"]=ii["_id"]
                    dealcus[i] = t
                else:
                    # if n0<n1:
                        # t={}
                    t["需处理"]=n1-n0
                    t["数据源总计"] = n1
                    t["现有"]=n0
                    t["maxid"]=None
                    for ii in db[ii].find({},{"_id":1}).sort([("_id",-1)]).limit(1):
                        t["maxid"]=ii["_id"]
                    dealcus[i] = t
        else:
            # n0 = db[i].count_documents({})
            n1 = dbk[i].count_documents({})
            t = {}
            t["需处理"] = n1
            t["数据源总计"]=n1
            t["现有"] = 0
            t["maxid"] = None
            dealcus[i] = t
        if t:
            print("待处理",i,t)
        else:
            print("不处理",i)
    if ischeck:
        return None
    for k,v in dealcus.items():
        if k in renames.keys():
            kk=renames[k]
        else:
            kk=k
        cu0 = dbk[k]
        cu1 = db[kk]
        batch_size=2000
        datas=[]
        count=0
        if v["maxid"]:
            if refresh_find:
                print(datetime.datetime.now(), "增量同步处理", k, v)
                n0=0
                n1=0
                for t in cu0.find(refresh_find).batch_size(batch_size):
                    id=t["_id"]
                    t["uptime"]=uptime
                    if calldeal:
                        t=calldeal(t)
                    if id <= v["maxid"]:
                        insert_one(cu1, t, updteflag=True, id=["timekey"])
                        n0 = n0 + 1
                    else:
                        insert_one(cu1, t)
                        n1 = n1 + 1
                print("更新",n0,"新增",n1)
            else:
                print(datetime.datetime.now(), "增量同步处理", k, v)
                for i in cu0.find({"_id":{"$gt":v["maxid"]}}).batch_size(batch_size):
                    i["uptime"] = uptime
                    if calldeal:
                        i=calldeal(i)
                    if isinsert_one:
                        insert_one(cu1,i)
                    else:
                        if count<batch_size:
                            datas.append(i)
                            count=count+1
                        elif count==batch_size:
                            cu1.insert_many(datas)
                            datas=[i]
                            count=1
                    # cu1.insert_one(i)
                if not isinsert_one and datas:
                    cu1.insert_many(datas)
                    datas = []
                    count = 0
        else:
            print(datetime.datetime.now(), "全量同步处理", k, v)
            cu1.drop()
            for i in cu0.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append((k1, int(v1)))
                    if "unique" in v.keys():
                        cu1.create_index(t, name=v["name"], unique=v["unique"])
                    else:
                        cu1.create_index(t, name=v["name"], unique=False)
            for i in cu0.find(refresh_find).batch_size(batch_size):
                # cu1.insert_one(i)
                i["uptime"] = uptime
                if calldeal:
                    i = calldeal(i)
                if isinsert_one:
                    insert_one(cu1, i)
                else:
                    if count < batch_size:
                        datas.append(i)
                        count = count + 1
                    elif count == batch_size:
                        cu1.insert_many(datas)
                        datas = [i]
                        count = 1
            if not isinsert_one and datas:
                cu1.insert_many(datas)
                datas = []
                count = 0
    client.close()
    clientk.close()
def mergedb(sorureurl,tourl,sourcedb,todb,onlydealcus=None,nodealcus=[],isfresh=False,refresh_cu=[],refresh_find={},ischeck=False,isinsert_one=False,renames={},calldeal=None):
    print("start")
    clientk = mc(sorureurl)
    dbk = clientk[sourcedb]
    client = mc(tourl)
    db = client[todb]
    cus=dbk.list_collection_names()
    print("from ",cus)
    tocus=db.list_collection_names()
    print("to ", tocus)
    dealcus={}
    uptime=datetime.datetime.now()
    for i in cus:
        if onlydealcus:
            if i not in onlydealcus:
                continue
        if i in nodealcus:
            continue
        if i in renames.keys():
            ii=renames[i]
        else:
            ii=i
        if i in tocus:
            t = {}
            n0 = db[ii].count_documents(refresh_find)
            n1 = dbk[i].count_documents(refresh_find)
            if n1!=0:
                if refresh_find:
                    isfresh0 = True
                elif i in refresh_cu:
                    isfresh0 = True
                else:
                    isfresh0 = isfresh
                if isfresh0:
                    # t = {}
                    t["需处理"] = n1
                    t["数据源总计"] = n1
                    t["现有"] = n0
                    t["maxid"] = None
                    if refresh_find:
                        for ii in db[ii].find({},{"_id":1}).sort([("_id",-1)]).limit(1):
                            t["maxid"]=ii["_id"]
                    dealcus[i] = t
                else:
                    t["需处理"]=n1
                    t["数据源总计"] = n1
                    t["现有"]=n0
                    t["maxid"]=None
                    for ii in db[ii].find({},{"_id":1}).sort([("_id",-1)]).limit(1):
                        t["maxid"]=ii["_id"]
                    dealcus[i] = t
        else:
            # n0 = db[i].count_documents({})
            n1 = dbk[i].count_documents({})
            t = {}
            t["需处理"] = n1
            t["数据源总计"]=n1
            t["现有"] = 0
            t["maxid"] = None
            dealcus[i] = t
        if t:
            print("待处理",i,t)
        else:
            print("不处理",i)
    if ischeck:
        return None
    for k,v in dealcus.items():
        if k in renames.keys():
            kk=renames[k]
        else:
            kk=k
        cu0 = dbk[k]
        cu1 = db[kk]
        batch_size=2000
        datas=[]
        count=0
        if v["maxid"]:
            if refresh_find:
                print(datetime.datetime.now(), "增量同步处理", k, v)
                n0=0
                n1=0
                for t in cu0.find(refresh_find).batch_size(batch_size):
                    id=t["_id"]
                    t["uptime"]=uptime
                    if calldeal:
                        t=calldeal(t)
                    if id <= v["maxid"]:
                        insert_one(cu1, t, updteflag=True, id=["timekey"])
                        n0 = n0 + 1
                    else:
                        insert_one(cu1, t)
                        n1 = n1 + 1
                print("更新",n0,"新增",n1)
            else:
                print(datetime.datetime.now(), "增量同步处理", k, v)
                for i in cu0.find({"_id":{"$gt":v["maxid"]}}).batch_size(batch_size):
                    i["uptime"] = uptime
                    if calldeal:
                        i=calldeal(i)
                    if isinsert_one:
                        insert_one(cu1,i)
                    else:
                        if count<batch_size:
                            datas.append(i)
                            count=count+1
                        elif count==batch_size:
                            cu1.insert_many(datas)
                            datas=[i]
                            count=1
                    # cu1.insert_one(i)
                if not isinsert_one and datas:
                    cu1.insert_many(datas)
                    datas = []
                    count = 0
        else:
            print(datetime.datetime.now(), "全量同步处理", k, v)
            cu1.drop()
            for i in cu0.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append((k1, int(v1)))
                    if "unique" in v.keys():
                        cu1.create_index(t, name=v["name"], unique=v["unique"])
                    else:
                        cu1.create_index(t, name=v["name"], unique=False)
            for i in cu0.find({}).batch_size(batch_size):
                # cu1.insert_one(i)
                i["uptime"] = uptime
                if calldeal:
                    i = calldeal(i)
                if isinsert_one:
                    insert_one(cu1, i)
                else:
                    if count < batch_size:
                        datas.append(i)
                        count = count + 1
                    elif count == batch_size:
                        cu1.insert_many(datas)
                        datas = [i]
                        count = 1
            if not isinsert_one and datas:
                cu1.insert_many(datas)
                datas = []
                count = 0
    client.close()
    clientk.close()
def copydb_drop_double(dburl,dbname,args,isdrop=False):
    """

    Args:
        sorureurl:
        tourl:
        sourcedb:
        args: cu  keys:[]

    Returns:

    """
    from pymongo import MongoClient as mc
    from jili.data.db import getdb_client, insert_one
    client= mc(dburl)
    db = client[dbname]
    cunames=db.list_collection_names()
    for cuname,keys in args.items():
        cuname0=cuname+"_copy"
        if cuname0 in cunames:
            print("请检查存在",cuname,cuname0)
            continue
        cu0=db[cuname]
        cu0.rename(cuname0)
        cu1=db[cuname]
        cu0=db[cuname0]
        n0 = cu0.count_documents({})
        n1 = cu1.count_documents({})
        for i in cu0.list_indexes():
            v = i.to_dict()
            if v["name"] != "_id_":
                t = []
                for k1, v1 in v["key"].items():
                    t.append((k1, int(v1)))
                if "unique" in v.keys():
                    cu1.create_index(t, name=v["name"], unique=v["unique"])
                else:
                    cu1.create_index(t, name=v["name"], unique=False)
        keys_all=[]

        for i in cu0.find({}).sort([("_id",-1)]):
            k=gen_key(i,keys)
            if k not in keys_all:
                keys_all.append(k)
                cu1.insert_one(i)
        n1=cu1.count_documents({})
        print(cuname,n0,n1,n0-n1)
        if isdrop:
            cu0.drop()
def copydb_index(sorureurl,tourl,sourcedb,todb,onlydealcus=None,nodealcus=[]):
    clientk = mc(sorureurl)
    db0 = clientk[sourcedb]
    client = mc(tourl)
    db1 = client[todb]
    cus = db0.list_collection_names()
    print("from ", cus)
    tocus = db1.list_collection_names()
    print("to ", tocus)
    for k in cus:
        if onlydealcus:
            if k not in onlydealcus:
                continue
        if k in nodealcus:
            continue
        cu0 = db0[k]
        cu1 = db1[k]
        if k not in tocus:
            for i in cu0.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append((k1, int(v1)))
                    if "unique" in v.keys():
                        cu1.create_index(t, name=v["name"], unique=v["unique"])
                    else:
                        cu1.create_index(t, name=v["name"], unique=False)
        else:
            keys0 = []
            for i in cu0.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append(k1)
                    if "unique" in v.keys():
                        t.append("0")
                    t.sort()
                    t0 = ""
                    for j in t:
                        t0 = t0 + "." + j
                    keys0.append(t0)
            keys1 = []
            for i in cu1.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append(k1)
                    if "unique" in v.keys():
                        t.append("0")
                    t.sort()
                    t0=""
                    for j in t:
                        t0=t0+"."+j
                    keys1.append(t0)
            for i in keys0:
                if i not in keys1:
                    print(k,i)
    client.close()
    clientk.close()
def deal_index(sorureurl,tourl,sourcedb,todb):
    clientk = mc(sorureurl)
    db0 = clientk[sourcedb]
    client = mc(tourl)
    db1 = client[todb]
    cus = db0.list_collection_names()
    print("from ", cus)
    tocus = db1.list_collection_names()
    print("to ", tocus)
    for k in tocus:
        if k in cus:
            cu0 = db0[k]
            cu1 = db1[k]
            keys0=[]
            for i in cu0.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append(k1)
                    if "unique" in v.keys():
                        t.append("0")
                    t.sort()
                    t0 = ""
                    for j in t:
                        t0 = t0 + "." + j
                    keys0.append(t0)
            keys1 = []
            for i in cu1.list_indexes():
                v = i.to_dict()
                if v["name"] != "_id_":
                    t = []
                    for k1, v1 in v["key"].items():
                        t.append(k1)
                    if "unique" in v.keys():
                        t.append("0")
                    t.sort()
                    t0=""
                    for j in t:
                        t0=t0+"."+j
                    keys1.append(t0)
            for i in keys0:
                if i not in keys1:
                    print(k,i)
    client.close()
    clientk.close()

